{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hindi NewsCrawler.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "EqSXhVKvfXlA"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "2MMYj8iMQBU7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZoPOVT5xRBR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip3 install news-please   #https://github.com/fhamborg/news-please"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FtiRZuhioVcs",
        "colab_type": "text"
      },
      "source": [
        "##Config\n",
        "\n",
        "save these configurations under /root/news-please-repo/config , in google colab when you open the files tab, go up one level, then create folder under the root directory name it \"news-please-repo\" then under it create folder named \"config\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EqSXhVKvfXlA",
        "colab_type": "text"
      },
      "source": [
        "### config.cfg\n",
        "\n",
        "save this cell to a file called /root/news-please-repo/config/config.cfg"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NIvXgX0IoXwR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# IMPORTANT\n",
        "# All variables get parsed to the correct python-types (if not other declared)!\n",
        "# So bools have to be True or False (uppercase-first),\n",
        "# Floats need dots . (not comma)\n",
        "# Ints are just normal ints\n",
        "# dicts need to be like this { key: value }\n",
        "# arrays need to be like this [ value1, value2, value3 ]\n",
        "# All values in dicts and arrays will also be parsed.\n",
        "# Everything that does not match any of the above criteria will be parsed as string.\n",
        "\n",
        "\n",
        "\n",
        "[Crawler]\n",
        "\n",
        "# GENERAL\n",
        "# -------\n",
        "\n",
        "# Crawling heuristics\n",
        "# Default Crawlers:\n",
        "# Possibilities: RecursiveCrawler, RecursiveSitemapCrawler, RssCrawler, SitemapCrawler, Download (./newsplease/crawler/spiders/-dir)\n",
        "# default: SitemapCrawler\n",
        "default = SitemapCrawler\n",
        "\n",
        "# default:\n",
        "# fallbacks = {\n",
        "#     \"RssCrawler\": None,\n",
        "#     \"RecursiveSitemapCrawler\": \"RecursiveCrawler\",\n",
        "#     \"SitemapCrawler\": \"RecursiveCrawler\",\n",
        "#     \"RecursiveCrawler\": None,\n",
        "#     \"Download\": None\n",
        "#     }\n",
        "fallbacks = {\n",
        "    \"RssCrawler\": None,\n",
        "    \"RecursiveSitemapCrawler\": \"RecursiveCrawler\",\n",
        "    \"SitemapCrawler\": \"RecursiveCrawler\",\n",
        "    \"RecursiveCrawler\": None,\n",
        "    \"Download\": None\n",
        "    }\n",
        "\n",
        "# Determines how many hours need to pass since the last download of a webpage\n",
        "# to be downloaded again by the RssCrawler\n",
        "# default: 6\n",
        "hours_to_pass_for_redownload_by_rss_crawler = 6\n",
        "\n",
        "\n",
        "\n",
        "# PROCESSES\n",
        "# ---------\n",
        "\n",
        "# Number of crawlers, that should crawl parallel\n",
        "# not counting in daemonized crawlers\n",
        "# default: 5\n",
        "number_of_parallel_crawlers = 5\n",
        "\n",
        "# Number of daemons, will be added to daemons.\n",
        "# default: 10\n",
        "number_of_parallel_daemons = 10\n",
        "\n",
        "\n",
        "\n",
        "# SPECIAL CASES\n",
        "# -------------\n",
        "\n",
        "# urls which end on any of the following file extensions are ignored for recursive crawling\n",
        "# default: \"(pdf)|(docx?)|(xlsx?)|(pptx?)|(epub)|(jpe?g)|(png)|(bmp)|(gif)|(tiff)|(webp)|(avi)|(mpe?g)|(mov)|(qt)|(webm)|(ogg)|(midi)|(mid)|(mp3)|(wav)|(zip)|(rar)|(exe)|(apk)|(css)\"\n",
        "ignore_file_extensions = \"(pdf)|(docx?)|(xlsx?)|(pptx?)|(epub)|(jpe?g)|(png)|(bmp)|(gif)|(tiff)|(webp)|(avi)|(mpe?g)|(mov)|(qt)|(webm)|(ogg)|(midi)|(mid)|(mp3)|(wav)|(zip)|(rar)|(exe)|(apk)|(css)\"\n",
        "\n",
        "# urls which match the following regex are ignored for recursive crawling\n",
        "# default: \"\"\n",
        "ignore_regex = \"\"\n",
        "\n",
        "# Crawl the sitemaps of subdomains (if sitemap is enabled)\n",
        "# If True, any SitemapCrawler will try to crawl on the sitemap of the given domain including subdomains instead of a domain's main sitemap.\n",
        "# e.g. if True, a SitemapCrawler to be started on https://blog.zeit.de will try to crawl on the sitemap listed in http://blog.zeit.de/robots.txt. If not found, it will fall back to the False setting.\n",
        "#      if False, a SitemapCrawler to be started on https://blog.zeit.de will try to crawl on the sitemap listed in http://zeit.de/robots.txt\n",
        "# default: True\n",
        "sitemap_allow_subdomains = True\n",
        "\n",
        "\n",
        "\n",
        "[Heuristics]\n",
        "\n",
        "# Enabled heuristics,\n",
        "# Currently:\n",
        "#    - og_type\n",
        "#    - linked_headlines\n",
        "#    - self_linked_headlines\n",
        "#    - is_not_from_subdomain (with this setting enabled, it can be assured that only pages that aren't from a subdomain are downloaded)\n",
        "#    - meta_contains_article_keyword\n",
        "#    - crawler_contains_only_article_alikes\n",
        "# (maybe not up-to-date, see ./newsplease/helper_classes/heursitics.py:\n",
        "#  Every method not starting with __ should be a heuristic, except is_article)\n",
        "# These heuristics can be overwritten by sitelist.json for each site\n",
        "# default: {\"og_type\": True, \"linked_headlines\": \"<=0.65\", \"self_linked_headlines\": \"<=0.56\"}\n",
        "enabled_heuristics = {\"og_type\": True, \"linked_headlines\": \"<=0.65\", \"self_linked_headlines\": \"<=0.56\"}\n",
        "\n",
        "# Heuristics can be combined with others\n",
        "# The heuristics need to have the same name as in enabled_heuristics\n",
        "# Possible condition-characters / literals are: (, ), not, and, or\n",
        "# All heuristics used here need to be enabled in enabled_heuristics as well!\n",
        "# Examples:\n",
        "#     \"og_type and (self_linked_headlines or linked_headlines)\"\n",
        "#     \"og_type\"\n",
        "# default: \"og_type and (linked_headlines or self_linked_headlines)\"\n",
        "pass_heuristics_condition = \"og_type and (linked_headlines or self_linked_headlines)\"\n",
        "\n",
        "# The maximum ratio of headlines divided by linked_headlines in a file\n",
        "\n",
        "# The minimum number of headlines in a file to check for the ratio\n",
        "# If less then this number are in the file, the file will pass the test.\n",
        "# default: 5\n",
        "min_headlines_for_linked_test = 5\n",
        "\n",
        "\n",
        "\n",
        "[Files]\n",
        "\n",
        "# GENERAL:\n",
        "# -------\n",
        "\n",
        "# Paths:\n",
        "# toggles relative paths to be relative to the start_processes.py script (True) or relative to this config file (False)\n",
        "# This does not work for this config's 'Scrapy' section which is always relative to the dir the start_processes.py script is called from\n",
        "# Default: True\n",
        "relative_to_start_processes_file = True\n",
        "\n",
        "\n",
        "\n",
        "# INPUT:\n",
        "# -----\n",
        "\n",
        "# Here you can specify the input JSON-Filename\n",
        "# default: sitelist.hjson\n",
        "url_input_file_name = sitelist.hjson\n",
        "\n",
        "\n",
        "\n",
        "# OUTPUT:\n",
        "# ------\n",
        "\n",
        "# Toggles whether leading './' or '.\\' from above local_data_directory should be removed when saving the path into the Database\n",
        "# True: ./data would become data\n",
        "# default: True\n",
        "working_path = \"/content/drive/My Drive/Hindi_News\"\n",
        "\n",
        "# Following Strings in the local_data_directory will be replaced: (md5 hashes have a standard length of 32 chars)\n",
        "#\n",
        "# %working_path                           = the path specified in OUTPUT[\"working_path\"]\n",
        "# %time_download(<code>)                  = current time at download; will be replaced with strftime(<code>) where <code> is a string, explained further here: http://strftime.org/\n",
        "# %time_execution(<code>)                 = current time at execution; will be replaced with strftime(<code>) where <code> is a string, explained further here: http://strftime.org/\n",
        "# %timestamp_download                     = current time at download; unix-timestamp\n",
        "# %timestamp_execution                    = current time at execution; unix-timestamp\n",
        "# %domain(<size>)                         = first <size> chars of the domain of the crawled file (e.g. zeit.de)\n",
        "# %appendmd5_domain(<size>)               = appends the md5 to %domain(<<size> - 32 (md5 length) - 1 (_ as separator)>) if domain is longer than <size>\n",
        "# %md5_domain(<size>)                     = first <size> chars of md5 hash of %domain\n",
        "# %full_domain(<size>)                    = first <size> chars of the domain including subdomains (e.g. panamapapers.sueddeutsche.de)\n",
        "# %appendmd5_full_domain(<size>)          = appends the md5 to %full_domain(<<size> - 32 (md5 length) - 1 (_ as separator)>) if full_domain is longer than <size>\n",
        "# %md5_full_domain(<size>)                = first <size> chars of md5 hash of %full_domain\n",
        "# %subdomains(<size>)                     = first <size> chars of the domain's subdomains\n",
        "# %appendmd5_subdomains(<size>)           = appends the md5 to %subdomains(<<size> - 32 (md5 length) - 1 (_ as separator)>) if subdomains is longer than <size>\n",
        "# %md5_subdomains(<size>)                 = first <size> chars of md5 hash of %subdomains\n",
        "# %url_directory_string(<size>)           = first <size> chars of the directories on the server (e.g. http://panamapapers.sueddeutsche.de/articles/56f2c00da1bb8d3c3495aa0a/ would evaluate to articles_56f2c00da1bb8d3c3495aa0a), no filename\n",
        "# %appendmd5_url_directory_string(<size>) = appends the md5 to %url_directory_string(<<size> - 32 (md5 length) - 1 (_ as separator)>) if url_directory_string is longer than <size>\n",
        "# %md5_url_directory_string(<size>)       = first <size> chars of md5 hash of %url_directory_string(<size>)\n",
        "# %url_file_name(<size>)                  = first <size> chars of the file name (without type) on the server (e.g. http://www.spiegel.de/wirtschaft/soziales/ttip-dokumente-leak-koennte-ende-der-geheimhaltung-markieren-a-1090466.html would evaluate to ttip-dokumente-leak-koennte-ende-der-geheimhaltung-markieren-a-1090466, No filenames (indexes) will evaluate to index\n",
        "# %md5_url_file_name(<size>)              = first <size> chars of md5 hash of %url_file_name\n",
        "# %max_url_file_name                      = first x chars of %url_file_name, so the entire savepath has a length of the max possible length for a windows file system (260 characters - 1 <NUL>)\n",
        "# %appendmd5_max_url_file_name            = appends the md5 to the first x - 32 (md5 length) - 1 (_ as separator) chars of %url_file_name if the entire savepath has a length longer than the max possible length for a windows file system (260 characters - 1 <NUL>)\n",
        "#\n",
        "# This path can be relative or absolute, though to be able to easily merge multiple data sets, it should be kept relative and consistent on all datasets.\n",
        "# To be able to use cleanup commands, it should also start with a static folder name like 'data'.\n",
        "#\n",
        "# default: %working_path/data/%time_execution(%Y)/%time_execution(%m)/%time_execution(%d)/%appendmd5_full_domain(32)/%appendmd5_url_directory_string(60)_%appendmd5_max_url_file_name_%timestamp_download.html\n",
        "local_data_directory = %working_path/data/%appendmd5_full_domain(32)/news%timestamp_download.html\n",
        "\n",
        "\n",
        "# Toggles whether leading './' or '.\\' from above local_data_directory should be removed when saving the path into the Database\n",
        "# True: ./data would become data\n",
        "# default: True\n",
        "format_relative_path = True\n",
        "\n",
        "\n",
        "\n",
        "[MySQL]\n",
        "\n",
        "# MySQL-Connection required for saving meta-informations\n",
        "host = localhost\n",
        "port = 3306\n",
        "db = 'news-please'\n",
        "username = 'root'\n",
        "password = 'password'\n",
        "\n",
        "\n",
        "\n",
        "[Elasticsearch]\n",
        "\n",
        "# Elasticsearch-Connection required for saving detailed meta-information\n",
        "host = localhost\n",
        "port = 9200\n",
        "index_current = 'news-please'\n",
        "index_archive = 'news-please-archive'\n",
        "\n",
        "# Elasticsearch supports user authentication by CA certificates. If your database is protected by certificate\n",
        "# fill in the following parameters, otherwise you can ignore them.\n",
        "use_ca_certificates = False\n",
        "ca_cert_path = /path/to/cacert.pem\n",
        "client_cert_path = /path/to/client_cert.pem\n",
        "client_key_path = /path/to/client_key.pem\n",
        "username = 'root'\n",
        "secret = 'password'\n",
        "\n",
        "# Properties of the document type used for storage.\n",
        "mapping = {\"properties\": {\n",
        "    \"url\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"source_domain\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"title_page\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"title_rss\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"localpath\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"filename\": {\"type\": \"keyword\"},\n",
        "    \"ancestor\": {\"type\": \"keyword\"},\n",
        "    \"descendant\": {\"type\": \"keyword\"},\n",
        "    \"version\": {\"type\": \"long\"},\n",
        "    \"date_download\": {\"type\": \"date\", \"format\":\"yyyy-MM-dd HH:mm:ss\"},\n",
        "    \"date_modify\": {\"type\": \"date\", \"format\":\"yyyy-MM-dd HH:mm:ss\"},\n",
        "    \"date_publish\": {\"type\": \"date\", \"format\":\"yyyy-MM-dd HH:mm:ss\"},\n",
        "    \"title\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"description\":  {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"text\": {\"type\": \"text\"},\n",
        "    \"authors\": {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"image_url\":  {\"type\": \"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\n",
        "    \"language\": {\"type\": \"keyword\"}\n",
        "    }}\n",
        "\n",
        "\n",
        "\n",
        "[ArticleMasterExtractor]\n",
        "\n",
        "# Choose which extractors you want to use.\n",
        "#\n",
        "# The Default is ['newspaper_extractor', 'readability_extractor', 'date_extractor', 'lang_detect_extractor'],\n",
        "# which are all integrated extractors right now.\n",
        "# Possibly extractors are 'newspaper_extractor' , 'readability_extractor' , 'date_extractor_extractor and 'lang_detect_extractor'\n",
        "# Examples: -Only Newspaper and date_extractor: extractors = ['newspaper', 'date_extractor']\n",
        "#           -Only Newspaper: extractors = ['newspaper']\n",
        "extractors = ['newspaper_extractor', 'readability_extractor', 'date_extractor', 'lang_detect_extractor']\n",
        "\n",
        "\n",
        "\n",
        "[DateFilter]\n",
        "\n",
        "# If added to the pipeline, this module provides the means to filter the extracted articles based on the publishing date.\n",
        "# Therefore this module has to be placed after the KM4 article extractor to access the publishing dates.\n",
        "#\n",
        "# All articles, with a publishing date outside of the given time interval are dropped. The dates used to specify the\n",
        "# time interval are included and should follow this format: 'yyyy-mm-dd hh:mm:ss'.\n",
        "#\n",
        "# It is also possible to only define one date, assigning the other variable the value 'None' to create an half-bounded\n",
        "# interval.\n",
        "\n",
        "start_date = '1999-01-01 00:00:00'\n",
        "end_date = '2999-12-31 00:00:00'\n",
        "\n",
        "# If 'True' articles without a publishing date are dropped.\n",
        "strict_mode = False\n",
        "\n",
        "\n",
        "\n",
        "[Scrapy]\n",
        "\n",
        "# Possible levels (must be UC-only): CRITICAL, ERROR, WARNING, INFO, DEBUG\n",
        "# default: WARNING\n",
        "LOG_LEVEL = INFO\n",
        "\n",
        "# logformat, see https://docs.python.org/2/library/logging.html#logrecord-attributes\n",
        "# default: [%(name)s:%(lineno)d|%(levelname)s] %(message)s\n",
        "LOG_FORMAT = [%(name)s:%(lineno)d|%(levelname)s] %(message)s\n",
        "\n",
        "# Can be a filename or None\n",
        "# default: None\n",
        "LOG_FILE = None\n",
        "\n",
        "LOG_DATEFORMAT = %Y-%m-%d %H:%M:%S\n",
        "\n",
        "LOG_STDOUT = False\n",
        "\n",
        "LOG_ENCODING = utf-8\n",
        "\n",
        "BOT_NAME = 'news-please'\n",
        "\n",
        "SPIDER_MODULES = ['newsplease.crawler.spiders']\n",
        "NEWSPIDER_MODULE = 'newsplease.crawler.spiders'\n",
        "\n",
        "# Resume/Pause functionality activation\n",
        "# default: .resume_jobdir\n",
        "JOBDIRNAME = .resume_jobdir\n",
        "\n",
        "# Respect robots.txt activation\n",
        "# default: True\n",
        "ROBOTSTXT_OBEY=True\n",
        "\n",
        "# Maximum number of concurrent requests across all domains\n",
        "# default: 16\n",
        "# IMPORTANT: This setting does not work since each crawler has its own scrapy instance, but it might limit the concurrent_requests_per_domain if said setting has a higher number set than this one.\n",
        "CONCURRENT_REQUESTS=16\n",
        "\n",
        "# Maximum number of active requests per domain\n",
        "# default: 4\n",
        "CONCURRENT_REQUESTS_PER_DOMAIN=4\n",
        "\n",
        "# User-agent activation\n",
        "# default: 'news-please (+http://www.example.com/)'\n",
        "USER_AGENT = 'news-please (+http://www.example.com/)'\n",
        "\n",
        "# Pipeline activation\n",
        "# Syntax: '<relative location>.<Pipeline name>': <Order of execution from 0-1000>\n",
        "# default: {'newsplease.pipeline.pipelines.ArticleMasterExtractor':100, 'newsplease.crawler.pipeline.HtmlFileStorage':200, 'newsplease.pipeline.pipelines.JsonFileStorage': 300}\n",
        "# Further options: 'newsplease.pipeline.pipelines.ElasticsearchStorage': 350\n",
        "ITEM_PIPELINES = {'newsplease.pipeline.pipelines.ArticleMasterExtractor':100,\n",
        "                  ##'newsplease.pipeline.pipelines.HtmlFileStorage':200,\n",
        "                  'newsplease.pipeline.pipelines.JsonFileStorage':300\n",
        "                  }\n",
        "\n",
        "[Pandas]\n",
        "file_name = \"PandasStorage\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_LYp4_Crfhij",
        "colab_type": "text"
      },
      "source": [
        "### sitelist.hjson\n",
        "\n",
        "save this cell to a file called /root/news-please-repo/config/sitelist.hjson"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eP4S5BoLlWVM",
        "colab_type": "text"
      },
      "source": [
        "here we scrap from 9 websites, to add new websites, these are good links that would prove helpful\n",
        "\n",
        "https://bharatdiscovery.org/india/%E0%A4%B8%E0%A4%AE%E0%A4%BE%E0%A4%9A%E0%A4%BE%E0%A4%B0_%E0%A4%AA%E0%A4%A4%E0%A5%8D%E0%A4%B0\n",
        "https://bharatdiscovery.org/india/%E0%A4%A8%E0%A4%88%E0%A4%A6%E0%A5%81%E0%A4%A8%E0%A4%BF%E0%A4%AF%E0%A4%BE\n",
        "https://bharatdiscovery.org/india/%E0%A4%86%E0%A4%9C\n",
        "https://www.naidunia.com/latest-news"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j6xAbovLocSK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This is a HJSON-File, so comments and so on can be used! See https://hjson.org/\n",
        "# Furthermore this is first of all the actual config file, but as default just filled with examples.\n",
        "{\n",
        "  # Every URL has to be in an array-object in \"base_urls\".\n",
        "  # The same URL in combination with the same crawler may only appear once in this array.\n",
        "  \"base_urls\" : [\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://www.livehindustan.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://www.jagran.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://aajtak.intoday.in/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"http://money.bhaskar.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"http://bhaskar.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://navbharattimes.indiatimes.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"http://naidunia.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://www.abplive.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    },\n",
        "\t{\n",
        "      # Start crawling from timesofindia\n",
        "      \"url\": \"https://hindi.indiatvnews.com/\",\n",
        "\n",
        "      # Overwrite the default crawler and use th RecursiveCrawler instead\n",
        "      \"crawler\": \"RecursiveCrawler\",\n",
        "\n",
        "      # Because this site is weirt, use the\n",
        "      # meta_contains_article_keyword-heuristic and disable all others because\n",
        "      # overwrite will merge the defaults from \"newscrawler.cfg\" with\n",
        "      # this\n",
        "      \"overwrite_heuristics\": {\n",
        "        \"meta_contains_article_keyword\": true,\n",
        "        \"og_type\": false,\n",
        "        \"linked_headlines\": false,\n",
        "        \"self_linked_headlines\": false\n",
        "      },\n",
        "      # Also state that in the condition, all heuristics used in the condition\n",
        "      # have to be activated in \"overwrite_heuristics\" (or default) as well.\n",
        "      \"pass_heuristics_condition\": \"meta_contains_article_keyword\"\n",
        "    }\n",
        "\t]\n",
        "}\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9yEXZmnWoeV3",
        "colab_type": "text"
      },
      "source": [
        "##Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8glo9SaJxdax",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!news-please"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAGw0ugDA0im",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}